Import necessary libraries for the experiment

import os
current_directory = os.getcwd()
print(current_directory)

/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification

import numpy
numpy.version.version

'1.22.0'

# importing libraries

from tensorflow import keras
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
from tensorflow.keras.preprocessing import text, sequence
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers, callbacks

from tensorflow.python.keras.models import Model, Input
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GRU, Dense, Input, LSTM, Embedding, Dropout, SpatialDropout1D, Activation, SimpleRNN
from tensorflow.keras.layers import Conv1D, Bidirectional, GlobalMaxPool1D, MaxPooling1D, BatchNormalization, Add, Flatten
from tensorflow.keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate


from tensorflow.keras.optimizers import Adam
#from tensorflow.keras.optimizers import SGD

# For custom metrics
import keras.backend as K
from keras.utils.vis_utils import plot_model
from keras.callbacks import EarlyStopping, ModelCheckpoint

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
plt.style.use('seaborn')

import seaborn as sns
from IPython.display import Image

from tqdm import tqdm
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer 
import os, re, csv, math, codecs
from nltk.tokenize import word_tokenize
import string
import gensim

sns.set_style("whitegrid")
np.random.seed(0)

# Install dependencies
!apt install graphviz
!pip install pydot pydot-ng
!echo "Double check with Python 3"
!python -c "import pydot"

2023-04-02 04:07:49.749013: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.
To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.

The operation couldn’t be completed. Unable to locate a Java Runtime that supports apt.
Please visit http://www.java.com for information on installing Java.

Requirement already satisfied: pydot in /Users/macbookpro/opt/anaconda3/lib/python3.9/site-packages (1.4.2)
Requirement already satisfied: pydot-ng in /Users/macbookpro/opt/anaconda3/lib/python3.9/site-packages (2.0.0)
Requirement already satisfied: pyparsing>=2.1.4 in /Users/macbookpro/opt/anaconda3/lib/python3.9/site-packages (from pydot) (3.0.4)
Double check with Python 3

Loading the necessary data files and data Preparation

df = pd.read_csv('/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification/githubissuedata.csv')
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 226163 entries, 0 to 226162
Data columns (total 17 columns):
 #   Column            Non-Null Count   Dtype 
---  ------            --------------   ----- 
 0   title             225152 non-null  object
 1   body              225866 non-null  object
 2   task              226163 non-null  int64 
 3   bug               226163 non-null  int64 
 4   documentation     226163 non-null  int64 
 5   duplicate         226163 non-null  int64 
 6   enhancement       226163 non-null  int64 
 7   good_first_issue  226163 non-null  int64 
 8   help_wanted       226163 non-null  int64 
 9   invalid           226163 non-null  int64 
 10  question          226163 non-null  int64 
 11  wontfix           226163 non-null  int64 
 12  gitalk            226163 non-null  int64 
 13  priority_medium   226163 non-null  int64 
 14  priority_high     226163 non-null  int64 
 15  feature_request   226163 non-null  int64 
 16  feature           226163 non-null  int64 
dtypes: int64(15), object(2)
memory usage: 29.3+ MB

df.head(10)

	title	body	task	bug	documentation	duplicate	enhancement	good_first_issue	help_wanted	invalid	question	wontfix	gitalk	priority_medium	priority_high	feature_request	feature
0	My Account Paid laptop 1440 resolution Updat...	Case:Distance between Registered email address...	0	1	0	0	0	0	0	1	0	0	0	0	0	0	0
1	How to fix sleepimpl warning when ECS credenti...	Prerequisites X Ive searched for previous sim...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0
2	Slider doesnt work on touch devices	DescriptionSlider should work dragging in tou...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0
3	add new labels	DescriptionAdd ui and logic to permanently ad...	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0
4	No lib sub folder in Boost folder	Hi I am following thishttps://www.mlpack.org/d...	0	0	0	0	0	0	1	0	1	0	0	0	0	0	0
5	Add license notice to CLI	The CLI is missing the license notice. Theres ...	0	0	0	0	0	0	0	0	0	0	0	0	0	0	1
6	Should show Powershell or AzureCLI code necess...	There is example output from Powershell and Az...	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0
7	tidboperator could not work with kubernetes 1.23	Bug ReportWhat version of Kubernetes are you ...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0
8	Match Live	x Implement game logic x Calculate results ba...	0	0	0	0	1	0	0	0	0	0	0	0	0	0	0
9	AngularBug Make current location widget more g...	If youve never submitted an issue to the SORMA...	0	1	0	0	0	0	0	0	0	0	0	0	0	0	0

# check missing values in the dataset
print('The dataset has', df.isna().sum().sum(), 'missing values in test data.')
# check any duplicate records in the dataset
print('The dataset has', df.duplicated().sum(), 'duplicates in train data.')

The dataset has 1308 missing values in test data.
The dataset has 46411 duplicates in train data.

# remove missing values in the dataset
df.dropna(inplace= True)
# remove all duplicate records in the dataset
df.drop_duplicates(inplace= True)

# check missing values in the dataset
print('The dataset has', df.isna().sum().sum(), 'missing values in test data.')
# check any duplicate records in the dataset
print('The dataset has', df.duplicated().sum(), 'duplicates in train data.')

The dataset has 0 missing values in test data.
The dataset has 0 duplicates in train data.

# spliting dataset to train and test
from sklearn.model_selection import train_test_split
train_df, test_df = train_test_split(df, test_size=0.2, random_state=25)

print("Train data shape", train_df.shape)
print("Test data shape", test_df.shape)

Train data shape (143145, 17)
Test data shape (35787, 17)

CORRELATION BETWEEN DATA LABELS

fig, ax = plt.subplots(figsize=(10, 6))
fig.suptitle('Correlation Matrix')
sns.heatmap(train_df.corr(), annot=True, cmap="Greens", linewidths=.5, ax=ax);

The correlation figure below shows that Toxic" comments are clearly correlated with both "obscene" and "insult" comments. Interestingly, "toxic" and "severe_toxic" are only weakly correlated. While we can also observe that, "Obscene" comments and "insult" comments are also highly correlated, which makes perfect sense.

DATA PREPROCESSING

Deep Neural Networks input layers make use of input variables to feed the network for training the model. But in this task (experiment), we're dealing with words text. How do we represent these words in order to feed our model?

In our experiment, we used densed representation of those text (comments) and their semanticity together. The advantage of using this approach is the best way for fitting neural networks onto a text data (as in our case), as well as less memory usage compared to other sparse representation approaches.

Word Embedding

Two ways to feed embeddings to neural networks:

• Using your own word embeddings by training
• Using pre-trained embedding (e.g Word2vec, lad2vec, Glove etc)

#Convert text to vectors using keras preprocessing library tools

X_train = train_df["body"].values
X_test  = test_df["body"].values

y_train = train_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values
y_test  = test_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values

For the first embedding, we used keras preprocessing (Text Preprocessing) libraries. This class allows to vectorize a text corpus, by turning each text into either a sequence of integers (each integer being the index of a token in a dictionary) or into a vector where the coefficient for each token could be binary, based on word count, based on tf-idf

num_words = 20000 #Max. words to use per issue label
max_features = 200000 #Max. number of unique words in embeddinbg vector
max_len = 500 #Max. number of words per issue to be use
embedding_dims = 128 #embedding vector output dimension 
num_epochs = 15 # (before 5)number of epochs (number of times that the model is exposed to the training dataset)
val_split = 0.1
batch_size2 = 256 #(before 32)The **batch size** is the number of training examples in one forward/backward pass.
                  # In general, larger batch sizes result in faster progress in training, but don't always converge as quickly. 
                  #Smaller batch sizes train slower, but can converge faster. And the higher the batch size, the more memory space you’ll need.

#Issue body Tokenization
tokenizer = tokenizer = Tokenizer(num_words)
tokenizer.fit_on_texts(list(X_train))

#Convert tokenized issue body text to sequnces
X_train = tokenizer.texts_to_sequences(X_train)
X_test = tokenizer.texts_to_sequences(X_test)
 
# padding the sequences
X_train = pad_sequences(X_train, max_len)
X_test  = pad_sequences(X_test,  max_len)

print('X_train shape:', X_train.shape)
print('X_test shape: ', X_test.shape)

X_train shape: (143145, 500)
X_test shape:  (35787, 500)

##We use cross validation to split arrays or matrices of train data into random train and validation subsets
X_tra, X_val, y_tra, y_val = train_test_split(X_train, y_train, train_size =0.9, random_state=233)

#We used early callback functionality that allows you to specify the performance measure to monitor, the trigger, and once triggered. It will stop the training process.
early = EarlyStopping(monitor="val_loss", mode="min", patience=4)

#Writing functions for Precision, Recall, F1-Measure, AUC, mean etc evaluaiton metrics to evaluate the model

#Import necessary libraries
4# demonstration of calculating metrics for a neural network model using sklearn
from sklearn.datasets import make_circles
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix


def precision(y_true, y_pred):
    #Calculating precision, a metric for multi-label classification of how many selected items are relevant.
    
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
    precision = true_positives / (predicted_positives + K.epsilon())
    return precision


def recall(y_true, y_pred):
    #Calculating recall, a metric for multi-label classification of how many relevant items are selected.
    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
    recall = true_positives / (possible_positives + K.epsilon())
    return recall

  #Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
    return K.mean(y_pred)

def fbeta_score(y_true, y_pred, beta=1):
    '''Calculates the F score, the weighted harmonic mean of precision and recall.
    This is useful for multi-label classification, where input samples can be
    classified as sets of labels. By only using accuracy (precision) a model
    would achieve a perfect score by simply assigning every class to every
    input. In order to avoid this, a metric should penalize incorrect class
    assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
    computes this, as a weighted mean of the proportion of correct class
    assignments vs. the proportion of incorrect class assignments.
    With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
    correct classes becomes more important, and with beta > 1 the metric is
    instead weighted towards penalizing incorrect class assignments.
    '''
    if beta < 0:
        raise ValueError('The lowest choosable beta is zero (only precision).')
        
    # If there are no true positives, fix the F score at 0 like sklearn.
    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
        return 0.0

    p = precision(y_true, y_pred)
    r = recall(y_true, y_pred)
    bb = beta ** 2
    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
    return fbeta_score

def auroc(y_true, y_pred):
    auc = tf.keras.metrics.AUC(y_true, y_pred)[1]
    #auc = tf.metrics.auc(y_true, y_pred)[1]
    K.get_session().run(tf.local_variables_initializer())
    return auc

def fmeasure(y_true, y_pred):
    #Calculates the f-measure, the harmonic mean of precision and recall.
    return fbeta_score(y_true, y_pred, beta=1)


fscore = f1score = fmeasure

Load the FastText Word Embeddng Corpus

#load embeddings
print('loading word embeddings...')
fastText_embeddings_index = {}
f = codecs.open('wiki.simple.vec', encoding='utf-8')
for line in tqdm(f):
    values = line.rstrip().rsplit(' ')
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    fastText_embeddings_index[word] = coefs
f.close()
print('found %s word vectors' % len(fastText_embeddings_index))

loading word embeddings...

111052it [00:11, 9927.93it/s]

found 111052 word vectors

Convert the word embedding into tokenized vector

import nltk
nltk.download('punkt')
nltk.download('stopwords')

issueBodies_lines = list()
lines = train_df['body'].values.tolist()

for line in lines:
    tokens = word_tokenize(line)
    
    #convert to lower case
    tokens = [w.lower() for w in tokens]
    
    #remove punctuation from each word
    table =  str.maketrans('', '', string.punctuation)
    stripped = [w.translate(table) for w in tokens]
    
    #remove remaining tpkens gthat are not alphabetic
    words = [word for word in stripped if word.isalpha()]
    
    #filter out stop words
    stop_words = set(stopwords.words('english'))
    words = [w for w in words if not w in stop_words]
    issueBodies_lines.append(words)


len(issueBodies_lines)

[nltk_data] Downloading package punkt to
[nltk_data]     /Users/macbookpro/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/macbookpro/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!

143145

#vectorize the text samples into a 2D integer tensor
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(issueBodies_lines)
sequences = tokenizer_obj.texts_to_sequences(issueBodies_lines)

#pad sequences
word_index = tokenizer_obj.word_index
print('Found %s uniquue tokens.' % len(word_index))

issueBodies_pad = pad_sequences(sequences, maxlen=max_len)
issue_tag = train_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values
print('Shape of issue bodies tensor', issueBodies_pad.shape)
print('Shape of issue bodies tensor', issue_tag.shape)

Found 792007 uniquue tokens.
Shape of issue bodies tensor (143145, 500)
Shape of issue bodies tensor (143145, 15)

Prepare the embedding matrix vectors in order to feed/pass the neural network

#embedding matrix
print('preparing embedding matrix...')
max_nb_words = 100000
fastText_embed_dim = 300 

words_not_found = []
nb_words = min(max_nb_words, len(word_index))
fastText_embedding_matrix = np.zeros((nb_words, fastText_embed_dim))
for word, i in word_index.items():
    if i >= nb_words:
        continue
    fastText_embedding_vector = fastText_embeddings_index.get(word)
    if (fastText_embedding_vector is not None) and len(fastText_embedding_vector) > 0:
        # words not found in embedding index will be all-zeros.
        fastText_embedding_matrix[i] = fastText_embedding_vector
    else:
        words_not_found.append(word)
print('number of null word embeddings: %d' % np.sum(np.sum(fastText_embedding_matrix, axis=1) == 0))

preparing embedding matrix...
number of null word embeddings: 80365

print(nb_words)

100000

#1 Convolutional Neural Network (CNN) with fastText
CNN_FastText_model = Sequential([
    Embedding(input_dim=fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
    SpatialDropout1D(0.5),
    # ... 100 filters with a kernel size of 4 so that each convolution will consider a window of 4 word embeddings
    Conv1D(filters=100, kernel_size=4, padding='same', activation='relu'),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    #It will be added after the activation function between a convolutional and a max-pooling layer.
    BatchNormalization(),
    GlobalMaxPool1D(),
    Dropout(0.5),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])

#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
    return K.mean(y_pred)

CNN_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

CNN_FastText_model.summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding (Embedding)       (None, 500, 300)          30000000  
                                                                 
 spatial_dropout1d (SpatialD  (None, 500, 300)         0         
 ropout1D)                                                       
                                                                 
 conv1d (Conv1D)             (None, 500, 100)          120100    
                                                                 
 batch_normalization (BatchN  (None, 500, 100)         400       
 ormalization)                                                   
                                                                 
 global_max_pooling1d (Globa  (None, 100)              0         
 lMaxPooling1D)                                                  
                                                                 
 dropout (Dropout)           (None, 100)               0         
                                                                 
 dense (Dense)               (None, 50)                5050      
                                                                 
 dense_1 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 30,126,315
Trainable params: 126,115
Non-trainable params: 30,000,200
_________________________________________________________________

CNN_FastText_model_fit = CNN_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])

Epoch 1/15
504/504 [==============================] - 327s 646ms/step - loss: 0.2166 - accuracy: 0.5015 - mean_pred: 0.0862 - fmeasure: 0.3444 - precision: 0.6240 - recall: 0.2450 - val_loss: 0.1876 - val_accuracy: 0.5667 - val_mean_pred: 0.0881 - val_fmeasure: 0.3272 - val_precision: 0.8626 - val_recall: 0.2024
Epoch 2/15
504/504 [==============================] - 319s 633ms/step - loss: 0.1814 - accuracy: 0.5644 - mean_pred: 0.0820 - fmeasure: 0.4403 - precision: 0.7137 - recall: 0.3193 - val_loss: 0.1746 - val_accuracy: 0.5969 - val_mean_pred: 0.0875 - val_fmeasure: 0.4871 - val_precision: 0.7437 - val_recall: 0.3626
Epoch 3/15
504/504 [==============================] - 319s 632ms/step - loss: 0.1771 - accuracy: 0.5831 - mean_pred: 0.0815 - fmeasure: 0.4741 - precision: 0.7072 - recall: 0.3576 - val_loss: 0.1711 - val_accuracy: 0.6039 - val_mean_pred: 0.0825 - val_fmeasure: 0.5201 - val_precision: 0.7187 - val_recall: 0.4080
Epoch 4/15
504/504 [==============================] - 347s 688ms/step - loss: 0.1746 - accuracy: 0.5941 - mean_pred: 0.0812 - fmeasure: 0.4936 - precision: 0.7098 - recall: 0.3796 - val_loss: 0.1691 - val_accuracy: 0.6151 - val_mean_pred: 0.0865 - val_fmeasure: 0.5398 - val_precision: 0.7121 - val_recall: 0.4351
Epoch 5/15
504/504 [==============================] - 320s 635ms/step - loss: 0.1723 - accuracy: 0.6004 - mean_pred: 0.0812 - fmeasure: 0.5047 - precision: 0.7106 - recall: 0.3926 - val_loss: 0.1659 - val_accuracy: 0.6196 - val_mean_pred: 0.0834 - val_fmeasure: 0.5554 - val_precision: 0.7235 - val_recall: 0.4512
Epoch 6/15
504/504 [==============================] - 354s 702ms/step - loss: 0.1701 - accuracy: 0.6061 - mean_pred: 0.0810 - fmeasure: 0.5135 - precision: 0.7175 - recall: 0.4010 - val_loss: 0.1643 - val_accuracy: 0.6155 - val_mean_pred: 0.0833 - val_fmeasure: 0.5494 - val_precision: 0.7305 - val_recall: 0.4407
Epoch 7/15
504/504 [==============================] - 321s 637ms/step - loss: 0.1686 - accuracy: 0.6094 - mean_pred: 0.0810 - fmeasure: 0.5183 - precision: 0.7217 - recall: 0.4056 - val_loss: 0.1618 - val_accuracy: 0.6362 - val_mean_pred: 0.0818 - val_fmeasure: 0.5379 - val_precision: 0.7578 - val_recall: 0.4173
Epoch 8/15
504/504 [==============================] - 319s 634ms/step - loss: 0.1675 - accuracy: 0.6112 - mean_pred: 0.0808 - fmeasure: 0.5214 - precision: 0.7249 - recall: 0.4086 - val_loss: 0.1625 - val_accuracy: 0.6247 - val_mean_pred: 0.0843 - val_fmeasure: 0.5335 - val_precision: 0.7632 - val_recall: 0.4105
Epoch 9/15
504/504 [==============================] - 318s 631ms/step - loss: 0.1665 - accuracy: 0.6148 - mean_pred: 0.0808 - fmeasure: 0.5275 - precision: 0.7247 - recall: 0.4158 - val_loss: 0.1611 - val_accuracy: 0.6370 - val_mean_pred: 0.0830 - val_fmeasure: 0.5549 - val_precision: 0.7451 - val_recall: 0.4425
Epoch 10/15
504/504 [==============================] - 320s 635ms/step - loss: 0.1656 - accuracy: 0.6176 - mean_pred: 0.0809 - fmeasure: 0.5312 - precision: 0.7266 - recall: 0.4197 - val_loss: 0.1612 - val_accuracy: 0.6301 - val_mean_pred: 0.0804 - val_fmeasure: 0.5404 - val_precision: 0.7661 - val_recall: 0.4179
Epoch 11/15
504/504 [==============================] - 319s 633ms/step - loss: 0.1649 - accuracy: 0.6202 - mean_pred: 0.0808 - fmeasure: 0.5347 - precision: 0.7290 - recall: 0.4231 - val_loss: 0.1598 - val_accuracy: 0.6357 - val_mean_pred: 0.0835 - val_fmeasure: 0.5726 - val_precision: 0.7246 - val_recall: 0.4737
Epoch 12/15
504/504 [==============================] - 312s 619ms/step - loss: 0.1644 - accuracy: 0.6213 - mean_pred: 0.0808 - fmeasure: 0.5381 - precision: 0.7281 - recall: 0.4278 - val_loss: 0.1601 - val_accuracy: 0.6328 - val_mean_pred: 0.0854 - val_fmeasure: 0.5667 - val_precision: 0.7469 - val_recall: 0.4570
Epoch 13/15
504/504 [==============================] - 323s 640ms/step - loss: 0.1637 - accuracy: 0.6237 - mean_pred: 0.0808 - fmeasure: 0.5435 - precision: 0.7268 - recall: 0.4348 - val_loss: 0.1584 - val_accuracy: 0.6397 - val_mean_pred: 0.0821 - val_fmeasure: 0.5758 - val_precision: 0.7348 - val_recall: 0.4738
Epoch 14/15
504/504 [==============================] - 314s 622ms/step - loss: 0.1632 - accuracy: 0.6254 - mean_pred: 0.0808 - fmeasure: 0.5465 - precision: 0.7286 - recall: 0.4379 - val_loss: 0.1595 - val_accuracy: 0.6320 - val_mean_pred: 0.0845 - val_fmeasure: 0.5855 - val_precision: 0.7038 - val_recall: 0.5016
Epoch 15/15
504/504 [==============================] - 315s 626ms/step - loss: 0.1629 - accuracy: 0.6271 - mean_pred: 0.0808 - fmeasure: 0.5482 - precision: 0.7269 - recall: 0.4411 - val_loss: 0.1576 - val_accuracy: 0.6433 - val_mean_pred: 0.0822 - val_fmeasure: 0.5795 - val_precision: 0.7392 - val_recall: 0.4769

# Plot training & validation accuracy values
plt.plot(CNN_FastText_model_fit.history['accuracy'])
plt.plot(CNN_FastText_model_fit.history['val_accuracy'])
plt.title('CNN-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(CNN_FastText_model_fit.history['loss'])
plt.plot(CNN_FastText_model_fit.history['val_loss'])
plt.title('CNN-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#2 Recurrent Neural Network (RNN) with fastText
RNN_FastText_model = Sequential([
    Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
    SpatialDropout1D(0.5),
    #Fully-connected RNN where the output is to be fed back to input.
    SimpleRNN(25, return_sequences=True),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    BatchNormalization(),
    Dropout(0.5),
    GlobalMaxPool1D(),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])

#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
    return K.mean(y_pred)

RNN_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

RNN_FastText_model.summary()

Model: "sequential_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_1 (Embedding)     (None, 500, 300)          30000000  
                                                                 
 spatial_dropout1d_1 (Spatia  (None, 500, 300)         0         
 lDropout1D)                                                     
                                                                 
 simple_rnn (SimpleRNN)      (None, 500, 25)           8150      
                                                                 
 batch_normalization_1 (Batc  (None, 500, 25)          100       
 hNormalization)                                                 
                                                                 
 dropout_1 (Dropout)         (None, 500, 25)           0         
                                                                 
 global_max_pooling1d_1 (Glo  (None, 25)               0         
 balMaxPooling1D)                                                
                                                                 
 dense_2 (Dense)             (None, 50)                1300      
                                                                 
 dense_3 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 30,010,315
Trainable params: 10,265
Non-trainable params: 30,000,050
_________________________________________________________________

RNN_FastText_model_fit = RNN_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])

Epoch 1/15
504/504 [==============================] - 199s 391ms/step - loss: 0.1996 - accuracy: 0.5055 - mean_pred: 0.0830 - fmeasure: 0.3450 - precision: 0.6646 - recall: 0.2442 - val_loss: 0.2351 - val_accuracy: 0.5438 - val_mean_pred: 0.1578 - val_fmeasure: 0.3763 - val_precision: 0.7679 - val_recall: 0.2496
Epoch 2/15
504/504 [==============================] - 198s 394ms/step - loss: 0.1868 - accuracy: 0.5263 - mean_pred: 0.0812 - fmeasure: 0.3700 - precision: 0.7290 - recall: 0.2490 - val_loss: 0.2108 - val_accuracy: 0.5425 - val_mean_pred: 0.1341 - val_fmeasure: 0.4122 - val_precision: 0.7260 - val_recall: 0.2881
Epoch 3/15
504/504 [==============================] - 197s 391ms/step - loss: 0.1840 - accuracy: 0.5410 - mean_pred: 0.0810 - fmeasure: 0.3944 - precision: 0.7293 - recall: 0.2720 - val_loss: 0.2088 - val_accuracy: 0.5751 - val_mean_pred: 0.1370 - val_fmeasure: 0.4300 - val_precision: 0.7677 - val_recall: 0.2990
Epoch 4/15
504/504 [==============================] - 197s 391ms/step - loss: 0.1802 - accuracy: 0.5601 - mean_pred: 0.0809 - fmeasure: 0.4341 - precision: 0.7165 - recall: 0.3128 - val_loss: 0.2038 - val_accuracy: 0.5809 - val_mean_pred: 0.1273 - val_fmeasure: 0.3956 - val_precision: 0.8226 - val_recall: 0.2609
Epoch 5/15
504/504 [==============================] - 198s 393ms/step - loss: 0.1792 - accuracy: 0.5653 - mean_pred: 0.0809 - fmeasure: 0.4425 - precision: 0.7208 - recall: 0.3203 - val_loss: 0.2021 - val_accuracy: 0.5868 - val_mean_pred: 0.1281 - val_fmeasure: 0.4608 - val_precision: 0.7668 - val_recall: 0.3298
Epoch 6/15
504/504 [==============================] - 199s 395ms/step - loss: 0.1784 - accuracy: 0.5683 - mean_pred: 0.0809 - fmeasure: 0.4498 - precision: 0.7198 - recall: 0.3283 - val_loss: 0.1910 - val_accuracy: 0.5785 - val_mean_pred: 0.1072 - val_fmeasure: 0.4530 - val_precision: 0.6723 - val_recall: 0.3419
Epoch 7/15
504/504 [==============================] - 196s 388ms/step - loss: 0.1955 - accuracy: 0.5013 - mean_pred: 0.0810 - fmeasure: 0.1991 - precision: 0.7152 - recall: 0.1574 - val_loss: 0.2093 - val_accuracy: 0.4957 - val_mean_pred: 0.1114 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068
Epoch 8/15
504/504 [==============================] - 198s 394ms/step - loss: 0.1970 - accuracy: 0.4926 - mean_pred: 0.0808 - fmeasure: 0.0900 - precision: 0.7832 - recall: 0.0664 - val_loss: 0.2134 - val_accuracy: 0.4957 - val_mean_pred: 0.1165 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068
Epoch 9/15
504/504 [==============================] - 199s 395ms/step - loss: 0.1970 - accuracy: 0.4926 - mean_pred: 0.0809 - fmeasure: 0.0921 - precision: 0.7828 - recall: 0.0685 - val_loss: 0.2068 - val_accuracy: 0.4957 - val_mean_pred: 0.1029 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068
Epoch 10/15
504/504 [==============================] - 197s 391ms/step - loss: 0.1969 - accuracy: 0.4926 - mean_pred: 0.0809 - fmeasure: 0.0835 - precision: 0.7927 - recall: 0.0603 - val_loss: 0.2064 - val_accuracy: 0.4957 - val_mean_pred: 0.1005 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068

# Plot training & validation accuracy values
plt.plot(RNN_FastText_model_fit.history['accuracy'])
plt.plot(RNN_FastText_model_fit.history['val_accuracy'])
plt.title('RNN-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(RNN_FastText_model_fit.history['loss'])
plt.plot(RNN_FastText_model_fit.history['val_loss'])
plt.title('RNN-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#3 LSTM with fastText

LSTM_FastText_model = Sequential([
    Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
    SpatialDropout1D(0.5),
    #Bidirectional layer will enable our model to predict a missing word in a sequence, 
    #So, using this feature will enable the model to look at the context on both the left and the right.
    LSTM(25, return_sequences=True),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    BatchNormalization(),
    Dropout(0.5),
    GlobalMaxPool1D(),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])


LSTM_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

2023-04-02 14:55:17.699192: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:55:17.701077: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:55:17.703066: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

Visualize the Orchestrated LSTM-fastText Model Architecture

LSTM_FastText_model.summary()

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_3 (Embedding)     (None, 500, 300)          30000000  
                                                                 
 spatial_dropout1d_3 (Spatia  (None, 500, 300)         0         
 lDropout1D)                                                     
                                                                 
 lstm_1 (LSTM)               (None, 500, 25)           32600     
                                                                 
 batch_normalization_3 (Batc  (None, 500, 25)          100       
 hNormalization)                                                 
                                                                 
 dropout_3 (Dropout)         (None, 500, 25)           0         
                                                                 
 global_max_pooling1d_3 (Glo  (None, 25)               0         
 balMaxPooling1D)                                                
                                                                 
 dense_6 (Dense)             (None, 50)                1300      
                                                                 
 dense_7 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 30,034,765
Trainable params: 34,715
Non-trainable params: 30,000,050
_________________________________________________________________

LSTM_FastText_model_fit = LSTM_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])

Epoch 1/15

2023-04-02 14:55:18.095303: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:55:18.097359: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:55:18.099749: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 14:55:19.058546: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:55:19.061164: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:55:19.063136: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

504/504 [==============================] - ETA: 0s - loss: 0.1861 - accuracy: 0.5581 - mean_pred: 0.0816 - fmeasure: 0.4369 - precision: 0.6876 - recall: 0.3245

2023-04-02 14:59:48.398786: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:59:48.400815: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:59:48.402662: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

504/504 [==============================] - 289s 568ms/step - loss: 0.1861 - accuracy: 0.5581 - mean_pred: 0.0816 - fmeasure: 0.4369 - precision: 0.6876 - recall: 0.3245 - val_loss: 0.2228 - val_accuracy: 0.6085 - val_mean_pred: 0.1569 - val_fmeasure: 0.5065 - val_precision: 0.7367 - val_recall: 0.3863
Epoch 2/15
504/504 [==============================] - 284s 563ms/step - loss: 0.1730 - accuracy: 0.5985 - mean_pred: 0.0810 - fmeasure: 0.4982 - precision: 0.7190 - recall: 0.3824 - val_loss: 0.2046 - val_accuracy: 0.6145 - val_mean_pred: 0.1415 - val_fmeasure: 0.5451 - val_precision: 0.6981 - val_recall: 0.4475
Epoch 3/15
504/504 [==============================] - 291s 577ms/step - loss: 0.1702 - accuracy: 0.6073 - mean_pred: 0.0810 - fmeasure: 0.5131 - precision: 0.7199 - recall: 0.3998 - val_loss: 0.2042 - val_accuracy: 0.6208 - val_mean_pred: 0.1398 - val_fmeasure: 0.5218 - val_precision: 0.7508 - val_recall: 0.4002
Epoch 4/15
504/504 [==============================] - 284s 564ms/step - loss: 0.1684 - accuracy: 0.6135 - mean_pred: 0.0810 - fmeasure: 0.5202 - precision: 0.7223 - recall: 0.4077 - val_loss: 0.1929 - val_accuracy: 0.6040 - val_mean_pred: 0.1236 - val_fmeasure: 0.5145 - val_precision: 0.7293 - val_recall: 0.3977
Epoch 5/15
504/504 [==============================] - 282s 560ms/step - loss: 0.1666 - accuracy: 0.6156 - mean_pred: 0.0810 - fmeasure: 0.5277 - precision: 0.7248 - recall: 0.4160 - val_loss: 0.1777 - val_accuracy: 0.6250 - val_mean_pred: 0.1056 - val_fmeasure: 0.4987 - val_precision: 0.7701 - val_recall: 0.3691
Epoch 6/15
504/504 [==============================] - 283s 561ms/step - loss: 0.1653 - accuracy: 0.6178 - mean_pred: 0.0810 - fmeasure: 0.5349 - precision: 0.7249 - recall: 0.4250 - val_loss: 0.1795 - val_accuracy: 0.6166 - val_mean_pred: 0.1066 - val_fmeasure: 0.5494 - val_precision: 0.7195 - val_recall: 0.4447
Epoch 7/15
504/504 [==============================] - 283s 561ms/step - loss: 0.1645 - accuracy: 0.6208 - mean_pred: 0.0809 - fmeasure: 0.5404 - precision: 0.7258 - recall: 0.4314 - val_loss: 0.1757 - val_accuracy: 0.6267 - val_mean_pred: 0.0984 - val_fmeasure: 0.4992 - val_precision: 0.7919 - val_recall: 0.3648
Epoch 8/15
504/504 [==============================] - 282s 560ms/step - loss: 0.1639 - accuracy: 0.6244 - mean_pred: 0.0809 - fmeasure: 0.5434 - precision: 0.7278 - recall: 0.4343 - val_loss: 0.1780 - val_accuracy: 0.6200 - val_mean_pred: 0.0997 - val_fmeasure: 0.5455 - val_precision: 0.7223 - val_recall: 0.4386
Epoch 9/15
504/504 [==============================] - 282s 559ms/step - loss: 0.1633 - accuracy: 0.6255 - mean_pred: 0.0809 - fmeasure: 0.5449 - precision: 0.7280 - recall: 0.4362 - val_loss: 0.1732 - val_accuracy: 0.6252 - val_mean_pred: 0.0950 - val_fmeasure: 0.5545 - val_precision: 0.7246 - val_recall: 0.4495
Epoch 10/15
504/504 [==============================] - 282s 559ms/step - loss: 0.1629 - accuracy: 0.6251 - mean_pred: 0.0809 - fmeasure: 0.5473 - precision: 0.7278 - recall: 0.4394 - val_loss: 0.1734 - val_accuracy: 0.6236 - val_mean_pred: 0.0881 - val_fmeasure: 0.5030 - val_precision: 0.7921 - val_recall: 0.3691
Epoch 11/15
504/504 [==============================] - 291s 578ms/step - loss: 0.1625 - accuracy: 0.6257 - mean_pred: 0.0809 - fmeasure: 0.5492 - precision: 0.7252 - recall: 0.4428 - val_loss: 0.1722 - val_accuracy: 0.6270 - val_mean_pred: 0.0908 - val_fmeasure: 0.5294 - val_precision: 0.7701 - val_recall: 0.4038
Epoch 12/15
504/504 [==============================] - 304s 602ms/step - loss: 0.1622 - accuracy: 0.6272 - mean_pred: 0.0808 - fmeasure: 0.5520 - precision: 0.7281 - recall: 0.4452 - val_loss: 0.1702 - val_accuracy: 0.6314 - val_mean_pred: 0.0860 - val_fmeasure: 0.5496 - val_precision: 0.7353 - val_recall: 0.4392
Epoch 13/15
504/504 [==============================] - 305s 606ms/step - loss: 0.1619 - accuracy: 0.6273 - mean_pred: 0.0809 - fmeasure: 0.5547 - precision: 0.7286 - recall: 0.4486 - val_loss: 0.1711 - val_accuracy: 0.6282 - val_mean_pred: 0.0862 - val_fmeasure: 0.5458 - val_precision: 0.7523 - val_recall: 0.4287
Epoch 14/15
504/504 [==============================] - 305s 605ms/step - loss: 0.1618 - accuracy: 0.6286 - mean_pred: 0.0809 - fmeasure: 0.5551 - precision: 0.7285 - recall: 0.4489 - val_loss: 0.1706 - val_accuracy: 0.6227 - val_mean_pred: 0.0864 - val_fmeasure: 0.5260 - val_precision: 0.7587 - val_recall: 0.4029
Epoch 15/15
504/504 [==============================] - 384s 762ms/step - loss: 0.1618 - accuracy: 0.6290 - mean_pred: 0.0809 - fmeasure: 0.5538 - precision: 0.7274 - recall: 0.4478 - val_loss: 0.1700 - val_accuracy: 0.6206 - val_mean_pred: 0.0836 - val_fmeasure: 0.5495 - val_precision: 0.7384 - val_recall: 0.4379

### Plot Training & Validation Accuracy with the Loss values of the LSTM-fastText Model# Plot training & validation accuracy values
plt.plot(LSTM_FastText_model_fit.history['accuracy'])
plt.plot(LSTM_FastText_model_fit.history['val_accuracy'])
plt.title('LSTM-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(LSTM_FastText_model_fit.history['loss'])
plt.plot(LSTM_FastText_model_fit.history['val_loss'])
plt.title('LSTM-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

4) Bidirecitional LSTM with fastText

#4 Bidirecitional LSTM with fastText
Bil_LSTM_FastText_model = Sequential([
    Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
    SpatialDropout1D(0.5),
    #Bidirectional layer will enable our model to predict a missing word in a sequence, 
    #So, using this feature will enable the model to look at the context on both the left and the right.
    Bidirectional(LSTM(25, return_sequences=True)),
    #**batch normalization layer** normalizes the activations of the previous layer at each batch, 
    #i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1. 
    BatchNormalization(),
    Dropout(0.5),
    GlobalMaxPool1D(),
    Dense(50, activation = 'relu'),
    Dense(15, activation = 'sigmoid')
])


Bil_LSTM_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])

2023-04-02 16:09:09.108807: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:09.110715: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:09.112385: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:09.286877: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:09.350579: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:09.352680: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:09.354569: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

Bil_LSTM_FastText_model.summary()

Model: "sequential_4"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 embedding_4 (Embedding)     (None, 500, 300)          30000000  
                                                                 
 spatial_dropout1d_4 (Spatia  (None, 500, 300)         0         
 lDropout1D)                                                     
                                                                 
 bidirectional (Bidirectiona  (None, 500, 50)          65200     
 l)                                                              
                                                                 
 batch_normalization_4 (Batc  (None, 500, 50)          200       
 hNormalization)                                                 
                                                                 
 dropout_4 (Dropout)         (None, 500, 50)           0         
                                                                 
 global_max_pooling1d_4 (Glo  (None, 50)               0         
 balMaxPooling1D)                                                
                                                                 
 dense_8 (Dense)             (None, 50)                2550      
                                                                 
 dense_9 (Dense)             (None, 15)                765       
                                                                 
=================================================================
Total params: 30,068,715
Trainable params: 68,615
Non-trainable params: 30,000,100
_________________________________________________________________

Bil_LSTM_FastText_model_fit = Bil_LSTM_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])

Epoch 1/15

2023-04-02 16:09:17.604221: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:17.606537: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:17.608798: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:17.781428: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:17.849680: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:17.851761: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:17.853625: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:18.572727: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:19.323679: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:19.326493: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:19.329047: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:19.497399: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:19.562745: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:19.565464: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:19.567264: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:20.235908: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]

504/504 [==============================] - ETA: 0s - loss: 0.1875 - accuracy: 0.5636 - mean_pred: 0.0822 - fmeasure: 0.4517 - precision: 0.6835 - recall: 0.3415

2023-04-02 16:17:46.164026: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:17:46.165725: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:17:46.167560: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:17:46.335129: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
	 [[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:17:46.392583: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
	 [[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:17:46.394523: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
	 [[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:17:46.396468: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
	 [[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]

504/504 [==============================] - 547s 1s/step - loss: 0.1875 - accuracy: 0.5636 - mean_pred: 0.0822 - fmeasure: 0.4517 - precision: 0.6835 - recall: 0.3415 - val_loss: 0.2214 - val_accuracy: 0.6109 - val_mean_pred: 0.1587 - val_fmeasure: 0.4878 - val_precision: 0.7495 - val_recall: 0.3619
Epoch 2/15
504/504 [==============================] - 527s 1s/step - loss: 0.1710 - accuracy: 0.6042 - mean_pred: 0.0811 - fmeasure: 0.5109 - precision: 0.7179 - recall: 0.3983 - val_loss: 0.2104 - val_accuracy: 0.6173 - val_mean_pred: 0.1499 - val_fmeasure: 0.5163 - val_precision: 0.7568 - val_recall: 0.3923
Epoch 3/15
504/504 [==============================] - 512s 1s/step - loss: 0.1670 - accuracy: 0.6174 - mean_pred: 0.0811 - fmeasure: 0.5320 - precision: 0.7207 - recall: 0.4230 - val_loss: 0.1967 - val_accuracy: 0.6232 - val_mean_pred: 0.1339 - val_fmeasure: 0.5334 - val_precision: 0.7448 - val_recall: 0.4159
Epoch 4/15
504/504 [==============================] - 521s 1s/step - loss: 0.1647 - accuracy: 0.6221 - mean_pred: 0.0810 - fmeasure: 0.5424 - precision: 0.7229 - recall: 0.4353 - val_loss: 0.1826 - val_accuracy: 0.6298 - val_mean_pred: 0.1190 - val_fmeasure: 0.5625 - val_precision: 0.7251 - val_recall: 0.4598
Epoch 5/15
504/504 [==============================] - 441s 876ms/step - loss: 0.1630 - accuracy: 0.6268 - mean_pred: 0.0810 - fmeasure: 0.5486 - precision: 0.7273 - recall: 0.4413 - val_loss: 0.1834 - val_accuracy: 0.6209 - val_mean_pred: 0.1188 - val_fmeasure: 0.5645 - val_precision: 0.7198 - val_recall: 0.4648
Epoch 6/15
504/504 [==============================] - 591s 1s/step - loss: 0.1616 - accuracy: 0.6310 - mean_pred: 0.0810 - fmeasure: 0.5561 - precision: 0.7275 - recall: 0.4510 - val_loss: 0.1775 - val_accuracy: 0.6342 - val_mean_pred: 0.1107 - val_fmeasure: 0.5636 - val_precision: 0.7309 - val_recall: 0.4590
Epoch 7/15
504/504 [==============================] - 566s 1s/step - loss: 0.1613 - accuracy: 0.6326 - mean_pred: 0.0811 - fmeasure: 0.5599 - precision: 0.7292 - recall: 0.4551 - val_loss: 0.1731 - val_accuracy: 0.6293 - val_mean_pred: 0.0998 - val_fmeasure: 0.5690 - val_precision: 0.7218 - val_recall: 0.4699
Epoch 8/15
504/504 [==============================] - 544s 1s/step - loss: 0.1602 - accuracy: 0.6352 - mean_pred: 0.0809 - fmeasure: 0.5627 - precision: 0.7347 - recall: 0.4568 - val_loss: 0.1716 - val_accuracy: 0.6371 - val_mean_pred: 0.0998 - val_fmeasure: 0.5758 - val_precision: 0.7292 - val_recall: 0.4761
Epoch 9/15
504/504 [==============================] - 491s 974ms/step - loss: 0.1595 - accuracy: 0.6375 - mean_pred: 0.0809 - fmeasure: 0.5649 - precision: 0.7331 - recall: 0.4603 - val_loss: 0.1705 - val_accuracy: 0.6377 - val_mean_pred: 0.0950 - val_fmeasure: 0.5689 - val_precision: 0.7384 - val_recall: 0.4631
Epoch 10/15
504/504 [==============================] - 522s 1s/step - loss: 0.1590 - accuracy: 0.6395 - mean_pred: 0.0809 - fmeasure: 0.5658 - precision: 0.7375 - recall: 0.4597 - val_loss: 0.1671 - val_accuracy: 0.6398 - val_mean_pred: 0.0892 - val_fmeasure: 0.5690 - val_precision: 0.7355 - val_recall: 0.4644
Epoch 11/15
504/504 [==============================] - 521s 1s/step - loss: 0.1585 - accuracy: 0.6407 - mean_pred: 0.0809 - fmeasure: 0.5692 - precision: 0.7371 - recall: 0.4643 - val_loss: 0.1699 - val_accuracy: 0.6432 - val_mean_pred: 0.0908 - val_fmeasure: 0.5788 - val_precision: 0.7322 - val_recall: 0.4789
Epoch 12/15
504/504 [==============================] - 496s 984ms/step - loss: 0.1578 - accuracy: 0.6434 - mean_pred: 0.0809 - fmeasure: 0.5725 - precision: 0.7379 - recall: 0.4683 - val_loss: 0.1698 - val_accuracy: 0.6409 - val_mean_pred: 0.0834 - val_fmeasure: 0.5735 - val_precision: 0.7382 - val_recall: 0.4692
Epoch 13/15
504/504 [==============================] - 481s 956ms/step - loss: 0.1578 - accuracy: 0.6435 - mean_pred: 0.0808 - fmeasure: 0.5731 - precision: 0.7371 - recall: 0.4694 - val_loss: 0.1718 - val_accuracy: 0.6385 - val_mean_pred: 0.0818 - val_fmeasure: 0.5664 - val_precision: 0.7423 - val_recall: 0.4583
Epoch 14/15
504/504 [==============================] - 476s 944ms/step - loss: 0.1577 - accuracy: 0.6457 - mean_pred: 0.0808 - fmeasure: 0.5752 - precision: 0.7392 - recall: 0.4712 - val_loss: 0.1734 - val_accuracy: 0.6454 - val_mean_pred: 0.0812 - val_fmeasure: 0.5868 - val_precision: 0.7276 - val_recall: 0.4921

# Plot training & validation accuracy values
plt.plot(Bil_LSTM_FastText_model_fit.history['accuracy'])
plt.plot(Bil_LSTM_FastText_model_fit.history['val_accuracy'])
plt.title('Bidirecitonal LSTM-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(Bil_LSTM_FastText_model_fit.history['loss'])
plt.plot(Bil_LSTM_FastText_model_fit.history['val_loss'])
plt.title('Bidirecitonal LSTM-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#5 Gated Recurrent (GRU) with fastText
sequence_input = Input(shape=(max_len, ))
model = Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False)(sequence_input)
model = SpatialDropout1D(0.2)(model)
model = GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1)(model)
model = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform")(model)
avg_pool = GlobalAveragePooling1D()(model)
max_pool = GlobalMaxPooling1D()(model)
model = concatenate([avg_pool, max_pool]) 
preds = Dense(15, activation="sigmoid")(model)
GRU_FastText_model = Model(sequence_input, preds)

GRU_FastText_model.compile(loss='binary_crossentropy',optimizer="Adam",metrics=['accuracy', mean_pred, fmeasure, precision, recall])

GRU_FastText_model.summary()

Model: "model"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
Total params: 30,191,695
Trainable params: 191,695
Non-trainable params: 30,000,000
__________________________________________________________________________________________________

GRU_FastText_model_fit = GRU_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])

Epoch 1/15
504/504 [==============================] - 1208s 2s/step - loss: 0.1855 - accuracy: 0.5680 - mean_pred: 0.0867 - fmeasure: 0.4439 - precision: 0.7107 - recall: 0.3294 - val_loss: 0.1684 - val_accuracy: 0.6199 - val_mean_pred: 0.0831 - val_fmeasure: 0.5356 - val_precision: 0.7314 - val_recall: 0.4229
Epoch 2/15
504/504 [==============================] - 1215s 2s/step - loss: 0.1628 - accuracy: 0.6320 - mean_pred: 0.0809 - fmeasure: 0.5434 - precision: 0.7489 - recall: 0.4276 - val_loss: 0.1614 - val_accuracy: 0.6424 - val_mean_pred: 0.0829 - val_fmeasure: 0.5405 - val_precision: 0.7725 - val_recall: 0.4162
Epoch 3/15
504/504 [==============================] - 1024s 2s/step - loss: 0.1560 - accuracy: 0.6531 - mean_pred: 0.0809 - fmeasure: 0.5707 - precision: 0.7584 - recall: 0.4584 - val_loss: 0.1566 - val_accuracy: 0.6583 - val_mean_pred: 0.0827 - val_fmeasure: 0.5755 - val_precision: 0.7566 - val_recall: 0.4648
Epoch 4/15
504/504 [==============================] - 1089s 2s/step - loss: 0.1512 - accuracy: 0.6671 - mean_pred: 0.0808 - fmeasure: 0.5899 - precision: 0.7670 - recall: 0.4801 - val_loss: 0.1538 - val_accuracy: 0.6662 - val_mean_pred: 0.0808 - val_fmeasure: 0.5903 - val_precision: 0.7543 - val_recall: 0.4853
Epoch 5/15
504/504 [==============================] - 979s 2s/step - loss: 0.1472 - accuracy: 0.6796 - mean_pred: 0.0808 - fmeasure: 0.6053 - precision: 0.7736 - recall: 0.4980 - val_loss: 0.1528 - val_accuracy: 0.6707 - val_mean_pred: 0.0788 - val_fmeasure: 0.5767 - val_precision: 0.7782 - val_recall: 0.4585
Epoch 6/15
504/504 [==============================] - 964s 2s/step - loss: 0.1434 - accuracy: 0.6898 - mean_pred: 0.0808 - fmeasure: 0.6197 - precision: 0.7834 - recall: 0.5136 - val_loss: 0.1513 - val_accuracy: 0.6732 - val_mean_pred: 0.0840 - val_fmeasure: 0.6106 - val_precision: 0.7490 - val_recall: 0.5157
Epoch 7/15
504/504 [==============================] - 963s 2s/step - loss: 0.1400 - accuracy: 0.6994 - mean_pred: 0.0808 - fmeasure: 0.6324 - precision: 0.7902 - recall: 0.5278 - val_loss: 0.1518 - val_accuracy: 0.6671 - val_mean_pred: 0.0790 - val_fmeasure: 0.5967 - val_precision: 0.7670 - val_recall: 0.4887
Epoch 8/15
504/504 [==============================] - 1031s 2s/step - loss: 0.1365 - accuracy: 0.7069 - mean_pred: 0.0808 - fmeasure: 0.6441 - precision: 0.7979 - recall: 0.5407 - val_loss: 0.1511 - val_accuracy: 0.6742 - val_mean_pred: 0.0836 - val_fmeasure: 0.6140 - val_precision: 0.7482 - val_recall: 0.5209
Epoch 9/15
504/504 [==============================] - 1104s 2s/step - loss: 0.1331 - accuracy: 0.7137 - mean_pred: 0.0808 - fmeasure: 0.6544 - precision: 0.8044 - recall: 0.5523 - val_loss: 0.1508 - val_accuracy: 0.6786 - val_mean_pred: 0.0824 - val_fmeasure: 0.6154 - val_precision: 0.7537 - val_recall: 0.5203
Epoch 10/15
504/504 [==============================] - 998s 2s/step - loss: 0.1298 - accuracy: 0.7236 - mean_pred: 0.0808 - fmeasure: 0.6659 - precision: 0.8125 - recall: 0.5647 - val_loss: 0.1537 - val_accuracy: 0.6734 - val_mean_pred: 0.0774 - val_fmeasure: 0.6114 - val_precision: 0.7580 - val_recall: 0.5127
Epoch 11/15
504/504 [==============================] - 975s 2s/step - loss: 0.1263 - accuracy: 0.7307 - mean_pred: 0.0807 - fmeasure: 0.6753 - precision: 0.8182 - recall: 0.5755 - val_loss: 0.1531 - val_accuracy: 0.6716 - val_mean_pred: 0.0849 - val_fmeasure: 0.6230 - val_precision: 0.7308 - val_recall: 0.5432
Epoch 12/15
504/504 [==============================] - 969s 2s/step - loss: 0.1230 - accuracy: 0.7378 - mean_pred: 0.0808 - fmeasure: 0.6860 - precision: 0.8246 - recall: 0.5878 - val_loss: 0.1546 - val_accuracy: 0.6706 - val_mean_pred: 0.0783 - val_fmeasure: 0.6178 - val_precision: 0.7480 - val_recall: 0.5267
Epoch 13/15
504/504 [==============================] - 973s 2s/step - loss: 0.1195 - accuracy: 0.7453 - mean_pred: 0.0807 - fmeasure: 0.6959 - precision: 0.8316 - recall: 0.5988 - val_loss: 0.1564 - val_accuracy: 0.6681 - val_mean_pred: 0.0807 - val_fmeasure: 0.6189 - val_precision: 0.7389 - val_recall: 0.5328

# Plot training & validation accuracy values
plt.plot(GRU_FastText_model_fit.history['accuracy'])
plt.plot(GRU_FastText_model_fit.history['val_accuracy'])
plt.title('Gated Recurrent Unit (GRU) with fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(GRU_FastText_model_fit.history['loss'])
plt.plot(GRU_FastText_model_fit.history['val_loss'])
plt.title('Bidirectional Gated Recurrent Unit (GRU) with fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

#6 Bidirectional Gated Recurrent (GRU) with fastText
sequence_input = Input(shape=(max_len, ))
model = Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False)(sequence_input)
model = SpatialDropout1D(0.2)(model)
model = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(model)
model = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform")(model)
avg_pool = GlobalAveragePooling1D()(model)
max_pool = GlobalMaxPooling1D()(model)
model = concatenate([avg_pool, max_pool]) 
preds = Dense(15, activation="sigmoid")(model)
Bil_GRU_FastText_model = Model(sequence_input, preds)

Bil_GRU_FastText_model.compile(loss='binary_crossentropy',optimizer="Adam",metrics=['accuracy', mean_pred, fmeasure, precision, recall])

Visualize the Orchestrated Bidirectional GRU+FastText Model Architecture

Bil_GRU_FastText_model.summary()

Model: "model_1"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
Total params: 30,381,391
Trainable params: 381,391
Non-trainable params: 30,000,000
__________________________________________________________________________________________________

Bil_GRU_FastText_model_fit = Bil_GRU_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])

Epoch 1/15
504/504 [==============================] - 1815s 4s/step - loss: 0.1802 - accuracy: 0.5886 - mean_pred: 0.0856 - fmeasure: 0.4758 - precision: 0.7180 - recall: 0.3633 - val_loss: 0.1641 - val_accuracy: 0.6319 - val_mean_pred: 0.0809 - val_fmeasure: 0.5255 - val_precision: 0.7631 - val_recall: 0.4011
Epoch 2/15
504/504 [==============================] - 1877s 4s/step - loss: 0.1591 - accuracy: 0.6429 - mean_pred: 0.0810 - fmeasure: 0.5585 - precision: 0.7554 - recall: 0.4441 - val_loss: 0.1575 - val_accuracy: 0.6562 - val_mean_pred: 0.0821 - val_fmeasure: 0.5509 - val_precision: 0.7838 - val_recall: 0.4252
Epoch 3/15
504/504 [==============================] - 1963s 4s/step - loss: 0.1521 - accuracy: 0.6650 - mean_pred: 0.0809 - fmeasure: 0.5867 - precision: 0.7668 - recall: 0.4759 - val_loss: 0.1535 - val_accuracy: 0.6653 - val_mean_pred: 0.0785 - val_fmeasure: 0.5911 - val_precision: 0.7572 - val_recall: 0.4851
Epoch 4/15
504/504 [==============================] - 2284s 5s/step - loss: 0.1466 - accuracy: 0.6806 - mean_pred: 0.0808 - fmeasure: 0.6080 - precision: 0.7769 - recall: 0.5003 - val_loss: 0.1514 - val_accuracy: 0.6743 - val_mean_pred: 0.0809 - val_fmeasure: 0.6068 - val_precision: 0.7547 - val_recall: 0.5077
Epoch 5/15
504/504 [==============================] - 2272s 5s/step - loss: 0.1418 - accuracy: 0.6950 - mean_pred: 0.0808 - fmeasure: 0.6264 - precision: 0.7879 - recall: 0.5206 - val_loss: 0.1502 - val_accuracy: 0.6765 - val_mean_pred: 0.0766 - val_fmeasure: 0.6078 - val_precision: 0.7608 - val_recall: 0.5066
Epoch 6/15
504/504 [==============================] - 2022s 4s/step - loss: 0.1367 - accuracy: 0.7065 - mean_pred: 0.0808 - fmeasure: 0.6433 - precision: 0.7978 - recall: 0.5397 - val_loss: 0.1501 - val_accuracy: 0.6742 - val_mean_pred: 0.0841 - val_fmeasure: 0.6140 - val_precision: 0.7509 - val_recall: 0.5197
Epoch 7/15
504/504 [==============================] - 2006s 4s/step - loss: 0.1315 - accuracy: 0.7189 - mean_pred: 0.0808 - fmeasure: 0.6602 - precision: 0.8077 - recall: 0.5588 - val_loss: 0.1497 - val_accuracy: 0.6771 - val_mean_pred: 0.0834 - val_fmeasure: 0.6204 - val_precision: 0.7449 - val_recall: 0.5319
Epoch 8/15
504/504 [==============================] - 1980s 4s/step - loss: 0.1261 - accuracy: 0.7318 - mean_pred: 0.0808 - fmeasure: 0.6753 - precision: 0.8172 - recall: 0.5760 - val_loss: 0.1531 - val_accuracy: 0.6745 - val_mean_pred: 0.0811 - val_fmeasure: 0.6133 - val_precision: 0.7510 - val_recall: 0.5186
Epoch 9/15
504/504 [==============================] - 1987s 4s/step - loss: 0.1205 - accuracy: 0.7433 - mean_pred: 0.0808 - fmeasure: 0.6919 - precision: 0.8284 - recall: 0.5946 - val_loss: 0.1525 - val_accuracy: 0.6710 - val_mean_pred: 0.0788 - val_fmeasure: 0.6110 - val_precision: 0.7510 - val_recall: 0.5155
Epoch 10/15
504/504 [==============================] - 1948s 4s/step - loss: 0.1149 - accuracy: 0.7569 - mean_pred: 0.0808 - fmeasure: 0.7084 - precision: 0.8379 - recall: 0.6142 - val_loss: 0.1560 - val_accuracy: 0.6729 - val_mean_pred: 0.0802 - val_fmeasure: 0.6188 - val_precision: 0.7432 - val_recall: 0.5305
Epoch 11/15
504/504 [==============================] - 1880s 4s/step - loss: 0.1088 - accuracy: 0.7699 - mean_pred: 0.0807 - fmeasure: 0.7266 - precision: 0.8489 - recall: 0.6358 - val_loss: 0.1604 - val_accuracy: 0.6590 - val_mean_pred: 0.0846 - val_fmeasure: 0.6157 - val_precision: 0.7158 - val_recall: 0.5407

# Plot training & validation accuracy values
plt.plot(Bil_GRU_FastText_model_fit.history['accuracy'])
plt.plot(Bil_GRU_FastText_model_fit.history['val_accuracy'])
plt.title(' Bidirectional Gated Recurrent Unit (GRU) with fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()

# Plot training & validation loss values
plt.plot(Bil_GRU_FastText_model_fit.history['loss'])
plt.plot(Bil_GRU_FastText_model_fit.history['val_loss'])
plt.title('Bidirectional Gated Recurrent Unit (GRU) with fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()

from chart_studio import plotly
import plotly.offline as py
import plotly.graph_objs as go

#Computing the highest of the evaluation matrics (per model)
trace = go.Table(
    header=dict(values=['Model', 'Loss', 'Accuracy', 'mean_pred', 'F-Measure', 'Precision', 'Recall'],
                line = dict(color='#7D7F80'),
                fill = dict(color='#a1c3d1'),
                align = ['left'] * 5),
    cells=dict(values=[['CNN-fastText', 'RNNs-fastText', 'LSTM-fastText', ' BiLSTM-fastText', 'GRU-fastText', 'BiGRU-fastText'],
                       [
                        #Loss Evaluation
                        round(np.max(CNN_FastText_model_fit.history['loss']), 3), round(np.max(RNN_FastText_model_fit.history['loss']), 3), 
                        round(np.max(LSTM_FastText_model_fit.history['loss']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['loss']), 3), round(np.max(GRU_FastText_model_fit.history['loss']), 3),
                        round(np.max(Bil_GRU_FastText_model_fit.history['loss']), 3)],
                        
                        #Accuracy Evaluation
                        [round(np.max(CNN_FastText_model_fit.history['accuracy']), 3), round(np.max(RNN_FastText_model_fit.history['accuracy']), 3), 
                        round(np.max(LSTM_FastText_model_fit.history['accuracy']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['accuracy']), 3), round(np.max(GRU_FastText_model_fit.history['accuracy']), 3),
                        round(np.max(Bil_GRU_FastText_model_fit.history['accuracy']), 3)],
               
                        #mean_pred Evaluation
                        [round(np.max(CNN_FastText_model_fit.history['mean_pred']), 3), round(np.max(RNN_FastText_model_fit.history['mean_pred']), 3), 
                        round(np.max(LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.max(GRU_FastText_model_fit.history['mean_pred']), 3),
                        round(np.max(Bil_GRU_FastText_model_fit.history['mean_pred']), 3)],
                        
                        #F1-Measure Evaluation fmeasure
                        [round(np.max(CNN_FastText_model_fit.history['fmeasure']), 3), round(np.max(RNN_FastText_model_fit.history['fmeasure']), 3), 
                        round(np.max(LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.max(GRU_FastText_model_fit.history['fmeasure']), 3),
                        round(np.max(Bil_GRU_FastText_model_fit.history['fmeasure']), 3)],
                                     
                        #Precision Evaluation precision
                        [round(np.max(CNN_FastText_model_fit.history['precision']), 3), round(np.max(RNN_FastText_model_fit.history['precision']), 3), 
                        round(np.max(LSTM_FastText_model_fit.history['precision']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['precision']), 3), round(np.max(GRU_FastText_model_fit.history['precision']), 3),
                        round(np.max(Bil_GRU_FastText_model_fit.history['precision']), 3)],
                       
                        #Recall Evaluation
                        [round(np.max(CNN_FastText_model_fit.history['recall']), 3), round(np.max(RNN_FastText_model_fit.history['recall']), 3), 
                        round(np.max(LSTM_FastText_model_fit.history['recall']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['recall']), 3), round(np.max(GRU_FastText_model_fit.history['recall']), 3),
                        round(np.max(Bil_GRU_FastText_model_fit.history['recall']), 3)]
                       ],
               line = dict(color='#7D7F80'),
               fill = dict(color='#EDFAFF'),
               align = ['left'] * 5))

layout = dict(width=800, height=400)
data = [trace]
fig = dict(data=data, layout=layout)
py.iplot(data, filename = 'FastTexttrained_embedding_with the max of the evaluation matrics (per model) _table')

#Computing the mean of the evaluation matrics (per model)
trace = go.Table(
    header=dict(values=['Model', 'Loss', 'Accuracy', 'mean_pred', 'F-Measure', 'Precision', 'Recall'],
                line = dict(color='#7D7F80'),
                fill = dict(color='#a1c3d1'),
                align = ['left'] * 5),
    cells=dict(values=[['CNN-fastText', 'RNNs-fastText', 'LSTM-fastText', ' BiLSTM-fastText', 'GRU-fastText', 'BiGRU-fastText'],
                       [
                        #Loss Evaluation
                        round(np.mean(CNN_FastText_model_fit.history['loss']), 3), round(np.mean(RNN_FastText_model_fit.history['loss']), 3), 
                        round(np.mean(LSTM_FastText_model_fit.history['loss']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['loss']), 3), round(np.mean(GRU_FastText_model_fit.history['loss']), 3),
                        round(np.mean(Bil_GRU_FastText_model_fit.history['loss']), 3)],
                        
                        #Accuracy Evaluation
                        [round(np.mean(CNN_FastText_model_fit.history['accuracy']), 3), round(np.mean(RNN_FastText_model_fit.history['accuracy']), 3), 
                        round(np.mean(LSTM_FastText_model_fit.history['accuracy']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['accuracy']), 3), round(np.mean(GRU_FastText_model_fit.history['accuracy']), 3),
                        round(np.mean(Bil_GRU_FastText_model_fit.history['accuracy']), 3)],
               
                        #mean_pred Evaluation
                        [round(np.mean(CNN_FastText_model_fit.history['mean_pred']), 3), round(np.mean(RNN_FastText_model_fit.history['mean_pred']), 3), 
                        round(np.mean(LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.mean(GRU_FastText_model_fit.history['mean_pred']), 3),
                        round(np.mean(Bil_GRU_FastText_model_fit.history['mean_pred']), 3)],
                        
                        #F1-Measure Evaluation fmeasure
                        [round(np.mean(CNN_FastText_model_fit.history['fmeasure']), 3), round(np.mean(RNN_FastText_model_fit.history['fmeasure']), 3), 
                        round(np.mean(LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.mean(GRU_FastText_model_fit.history['fmeasure']), 3),
                        round(np.mean(Bil_GRU_FastText_model_fit.history['fmeasure']), 3)],
                                     
                        #Precision Evaluation precision
                        [round(np.mean(CNN_FastText_model_fit.history['precision']), 3), round(np.mean(RNN_FastText_model_fit.history['precision']), 3), 
                        round(np.mean(LSTM_FastText_model_fit.history['precision']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['precision']), 3), round(np.mean(GRU_FastText_model_fit.history['precision']), 3),
                        round(np.mean(Bil_GRU_FastText_model_fit.history['precision']), 3)],
                        
                       
                        #Recall Evaluation
                        [round(np.mean(CNN_FastText_model_fit.history['recall']), 3), round(np.mean(RNN_FastText_model_fit.history['recall']), 3), 
                        round(np.mean(LSTM_FastText_model_fit.history['recall']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['recall']), 3), round(np.mean(GRU_FastText_model_fit.history['recall']), 3),
                        round(np.mean(Bil_GRU_FastText_model_fit.history['recall']), 3)]
                       ],
               line = dict(color='#7D7F80'),
               fill = dict(color='#EDFAFF'),
               align = ['left'] * 5))

layout = dict(width=800, height=400)
data = [trace]
fig = dict(data=data, layout=layout)
py.iplot(data, filename = 'FastTexttrained_embedding_with the Mean value of the evaluation matrics (per model) _table')

Score Confusion Table

#Score Confusion Table
# concat all training, validation and testing accuracy scores

CNN_FastTextAccuracy = ['Convolutional Neural Network (CNN) with fastText', 
                np.max(CNN_FastTextmodel_fit.history['accuracy']), 
                np.max(CNN_FastTextmodel_fit.history['val_accuracy']), 
                CNN_FastTexttest_score[1]]

RNN_FastTextAccuracy = ['Recurrent Neural Networks (RNNs) with fastText', 
                np.max(RNN_FastTextmodel_fit.history['accuracy']), 
                np.max(RNN_FastTextmodel_fit.history['val_accuracy']), 
                RNN_FastTexttest_score[1]]

LSTM_FastTextAccuracy = ['LSTM with fastText', 
                  np.max(LSTM_FastTextmodel_fit.history['accuracy']), 
                  np.max(LSTM_FastTextmodel_fit.history['val_accuracy']), 
                  LSTM_FastTexttest_score[1]]

Bidirectional_LSTM_FastTextAccuracy = ['Bidirectional LSTM with fastText', 
                    np.max(Bil_LSTM_FastTextmodel_fit.history['accuracy']), 
                    np.max(Bil_LSTM_FastTextmodel_fit.history['val_accuracy']), 
                    Bil_LSTM_FastTexttest_score[1]]

GRU_FastTextAccuracy = ['GRU with fastText', 
                    np.max(GRU_FastTextmodel_fit.history['accuracy']), 
                    np.max(GRU_FastTextmodel_fit.history['val_accuracy']), 
                    GRU_FastTexttest_score[1]]

Bidirectional_GRU_FastTextAccuracy = ['Bidirectional GRU with fastText', 
                    np.max(Bil_GRU_FastTextmodel_fit.history['accuracy']), 
                    np.max(Bil_GRU_FastTextmodel_fit.history['val_accuracy']), 
                    Bil_GRU_FastTexttest_score[1]]


# create dataframe
experimentalResult = pd.DataFrame([CNN_FastTextAccuracy])
# append all other scores
experimentalResult = experimentalResult.append([CNN_FastTextAccuracy, RNN_FastTextAccuracy, LSTM_FastTextAccuracy, Bidirectional_LSTM_FastTextAccuracy, 
                                                GRU_FastTextAccuracy, Bidirectional_GRU_FastTextAccuracy])


# beautify the new dataframe
experimentalResult.columns = ['Model', 'Training Accuracy', 'Validation Accuracy', 'Testing Accuracy']
experimentalResult.set_index(['Model'], inplace=True)
experimentalResult
print(experimentalResult)